#include <fstream>
#include <string>

using namespace std;

typedef map<string,int> Wordmap;
typedef map<string,float> Probmap;
typedef map<int,int> Countmap;
typedef set<string> Vocab;

struct Ngram {
	Wordmap ngram;
	Countmap gramcounts;
	int dim;
	Ngram(int _dim) : dim(_dim) { };
};

class LMgen {
	ifstream corpus;
	ifstream testcorp;
	vector<string> symtbl;
	vector<Ngram*> xgrams;
	Wordmap symtblidx;
	
	Vocab voc;
	Vocab missvoc;
	int misscnt;
	int unseen;
	bool map2unk;
	int unkidx;

	//Window-related
	int* l;
	int size;
	void incrementWindow();
	bool isCompleteWindow(int n);

	int corpsize; // N
	string getToken(istream& input);
	
	//Conversions
	string idx2string(int n);

	//Probabilities
	float m_bbi;
	float m_buni;
	Wordmap nvcount;
	Wordmap bigramprob;
	float bbi();
	float buni();
	float ball(int n);

public:
	LMgen(int n);
	~LMgen();

	void loadVocab(string filename);
	void loadCorpus(string filename);
	bool loadBigramProb(string filename);
	
	//generating maps
	int processWords();
	void genNgram(int n);
	void genCountcnt(int n);

	//getter
	int getCorpusSize() { return corpsize; };

	//output - TODO
	void dumpNgrams(int n);

	//Probabilities
	float pbi(string w, string v);
	float puni(string w);
	void genNvCount();

	void checkPuni();
	void checkBigramThe();

	//Perplexity
	float calcPP(string filename);
};
